# 抓取所有的房产信息
# 1.区 R1-R14
# 2.翻页 RG1---结尾

import requests
from lxml import etree
from bs4 import BeautifulSoup
import json

import pymongo
import redis


class MaiTianSpider(object):
    def __init__(self):
        self.url = 'http://bj.maitian.cn/esfall/R2/PG'
        self.headers = {
            "User-Agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_12_6) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/69.0.3497.100 Safari/537.36"
        }

        self.data_list = []
        self.page = 0

    def get_data(self, url):
        data = requests.get(url, headers=self.headers).content.decode()
        return data

    def save_data(self):
        json.dump(self.data_list, open('maitian.json', 'w'))
        # with open("maitian.html", 'w') as f:
        #     f.write(data)


        # 1.存数据库 mongodb redis
        # mongo = pymongo.MongoClient()
        # collection = mongo.maitian.room
        # collection.insert(self.data_list)
        # mongo.close()

        # redis_client = redis.StrictRedis()
        # redis_client.lpush("room", self.data_list)

    # 数据解析
    def parse_data(self, data):
        parse_data = BeautifulSoup(data, 'lxml')

        # 每个页面的 房子
        room_list = parse_data.select(".list_title")

        # 遍历每个房子的信息
        for room in room_list:
            room_dict = {}
            room_dict['room_title'] = room.select_one('h1 a').get_text()
            room_dict['room_detail_url'] = "http://bj.maitian.cn" + room.select_one('h1 a').get('href')
            room_dict['room_info'] = room.select_one('p').get_text()
            room_dict['room_five'] = room.select_one('dl').get_text()
            room_dict['room_price'] = room.select_one('div ol').get_text()

            self.data_list.append(room_dict)

    def start(self):

        # 动态解析 尾页 的 内容 把 539 取出来 在遍历
        # for page in range(1, 539):
        #     url = self.url + str(page)
        #     print(url)
        #     data = self.get_data(url)
        #     self.parse_data(data)
        # self.save_data()


        while True:
            self.page += 1
            url = self.url + str(self.page)
            print(url)
            data = self.get_data(url)
            self.parse_data(data)

            if data is None:
                break


        self.save_data()


spider = MaiTianSpider()
spider.start()

# xpath解析数据
"""
    parse_data = etree.HTML(data)

        # 取出每页 所有的房子
        room_list = parse_data.xpath('//div[@class="list_wrap"]/ul/li/div[@class="list_title"]')

        # 取出每个房子的信息
        for room in room_list:
            room_dict = {}
            room_dict['room_title'] = room.xpath('h1/a/text()')[0]
            room_dict['room_detail_url'] = 'http://bj.maitian.cn' +room.xpath('h1/a/@href')[0]
            room_dict['room_info'] = room.xpath('./p//text()')
            room_dict['room_five'] = room.xpath('dl/dd/mark/text()')
            room_dict['room_price'] = room.xpath('div/ol//text()')

            print(room_dict)
            
            
            正则匹配 r'\r\n' '\s' sub('',string)
            
"""
